#ifndef _TOKENIZER_H
#define _TOKENIZER_H

#include "common.h"
#include "list.h"


#define MAX_TOKEN_LENGTH 256

enum value_type {VALUE_CHAR = 0, VALUE_BYTE, VALUE_INT, VALUE_UINT, VALUE_FLOAT, VALUE_DOUBLE, VALUE_STRING};

#if 0
struct value_t {
	enum value_type type;
	union {
		char c;
		byte b;
		int i;
		uint ui;
		float f;
		double d;
		char* s;
	};
};
#endif

enum token_type {
		TOKEN_WHITESPACE = 0, TOKEN_INT, TOKEN_FLOAT, TOKEN_LBRACKET,
		TOKEN_RBRACKET, TOKEN_TYPE, TOKEN_STRING_INT, TOKEN_STRING_FLOAT,
		TOKEN_STRING_STRUCT, TOKEN_UNKNOWN, TOKEN_MAX};

struct token_t {
	enum token_type type;
	char string[MAX_TOKEN_LENGTH];
};


enum token_feedback {FEEDBACK_ERROR = 0, FEEDBACK_NEEDMORE, FEEDBACK_DONTCARE, FEEDBACK_DONE};

enum token_handler_state {STATE_ACTIVE = 0, STATE_INACTIVE};



struct token_handler_t {
	enum token_type type;
	enum token_feedback (*handler)(const char* const, int);
	enum token_handler_state state;
};


struct token_hit_t {
	int length;
	struct token_handler_t* handler;
};

struct script_holder_t {
	char* data;

	int start, count, length;
};

struct list_t* tokenize(const char* const filename);
void token_list_print(struct list_t* token_list);
void token_list_free(struct list_t** token_list);

#endif

